#!usr/bin/python
#coding=utf-8

import chardet
import urllib.request
import urllib.error
import socket
import gzip


from io import StringIO
from io import BytesIO

import urllib.parse
import string


#解码html内容
def decodeHtml(html):
    inf = chardet.detect(html)
    charset = inf['encoding']
    print(inf)
    newcharset = charset
    print(charset)
    if charset == 'gb2312' or charset == 'GB2312':
        newcharset = 'gbk'
    elif charset == None:
        print('charset none')

    try:
        html = html.decode(newcharset)
    except Exception as e:
        print('..................................................................')
        print(e)
        return None
        
    return html


#探测html编码格式
def htmlCoding(html):
    inf = chardet.detect(html)
    return inf['encoding']


#对url中的中文字符编码
def quoteUrl(url):
    return urllib.parse.quote(url, safe=string.printable)


'''
通过url来获取并解码内容
params 参数，决定get或post方式
headers 伪装浏览器  headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '  
                        'Chrome/51.0.2704.63 Safari/537.36'
    }
'''
def decodeHtmlByUrl(url, params={}, header={}):

    #中文编码
    url = quoteUrl(url)

    request = None
    
    if params:
        paramscoded = urllib.request.urlencode(params)
        if header:
            request = urllib.request.Request(url=url, data=paramscoded, headers = header)
        else:
            request = urllib.request.Request(url=url, data=paramscoded)
    else:
        if header:
            request = urllib.request.Request(url = url, headers=header)
        else:
            request = urllib.request.Request(url=url)

    request.add_header('Accept-encoding', 'gzip')

    try:
        response = urllib.request.urlopen(request, timeout=10)
        content= None
        if response.info().get('Content-Encoding', "") == 'gzip':
            buf = BytesIO(response.read())
            f = gzip.GzipFile(fileobj=buf)
            content = f.read()
        else:
            content = response.read()
        content = decodeHtml(content)
        return content
    except urllib.error.HTTPError as e:
        print(e.code)
        return None
    except socket.timeout as e:
        print("timeout.....")
        return None
    except Exception as e:
        print("other exception ................................ ")
        print(e)
        return None
    




'''
def decodeHtmlByUrl(url, params={}):
    if params:
        paramsencoded = urllib.request.urlencode()
    else:
        try:
            request = urllib.request.Request(url)
            request.add_header('Accept-encoding', 'gzip')
            response = urllib.request.urlopen(request, timeout=10)
            if response.info().get('Content-Encoding', "") == 'gzip':
                buf = BytesIO(response.read())
                f = gzip.GzipFile(fileobj = buf)
                content = f.read()
            else:
                content = response.read()
            content = decodeHtml(content)
            return content
        except urllib.error.HTTPError as e:
            print(e.code)
            return None
        except socket.timeout as e:
            print('timeout ............. ')
            return None
'''


